/*
@STI LICENSE
(C) 2006-2008 Universit�t Innsbruck.

This file is part of myOntology.

myOntologyis free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

myOntologyis distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with myOntology.  If not, see <http://www.gnu.org/licenses/>.

@STI LICENSE */
/**
 * 
 */
package org.myontology.importer;

import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.GregorianCalendar;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;

import model.Vocabulary;
import model.dataContainer.TranslationContainer;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.log4j.Logger;
import org.jdom.Attribute;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;
import org.jdom.xpath.XPath;
import org.openrdf.model.URI;
import org.openrdf.model.impl.LiteralImpl;
import org.openrdf.model.impl.URIImpl;
import org.openrdf.model.vocabulary.RDF;
import org.openrdf.query.QueryLanguage;
import org.openrdf.query.TupleQuery;
import org.openrdf.query.TupleQueryResult;
import org.openrdf.repository.RepositoryConnection;
import org.openrdf.repository.RepositoryException;

import controller.HelperFunctions;
import controller.elementMapper.ClassMapper;
import controller.elementMapper.OntologyModuleMapper;
import controller.sesameDataConnector.AbstractDataConnector;

/**
 *  @author <a href="michael.hackl@sti2.at">Michael Hackl</a>
 * Project: Myontology - www.myontology.org
 * STI
 */
public class XmlLoader {

	private RepositoryConnection connection;
	private URI currentSubject;
	private URI icecat;
	private URI currentOntology;
	
	public static Logger logger = Logger.getLogger(XmlLoader.class);
	
	public XmlLoader() {
		try {
			connection = AbstractDataConnector.getStandardConnection();
			OntologyModuleMapper mapper = new OntologyModuleMapper();
			icecat = createElement(connection, Vocabulary.ONTOLOGYMODULE, "IceCat");
			if (icecat!=null)
				mapper.insertLabel(connection, icecat, "Icecat Ontology Module", Vocabulary.ENGLISH);
		} catch (Exception e) {
			logger.error(e);
		}
		
	}
	
	/**
	 * @param args
	 */
	public static void main(String[] args) {
		new XmlLoader().load();
	}
	
	public void load() {
		
		
		
		
		HttpClient client = getLoggedInClient();


	        GetMethod get = new GetMethod("http://data.icecat.biz/export/freexml/");

	        get.setDoAuthentication( true );
	        Element root=null;
	        Element catRoot = null;
	        try {
	            // execute the GET
	            int status = client.executeMethod( get );

	            // print the status and response
	            String resultXML  =get.getResponseBodyAsString();
	            logger.debug(status + "\n"+resultXML );
	            SAXBuilder builder = new SAXBuilder();
	            Document doc = builder.build(new StringReader(resultXML));
	            root = doc.getRootElement();
	            
	            catRoot = getCategoryElement();

	        }catch (Exception e) {
	        	logger.error(e);
	        }finally {
	        
	            // release any connection resources used by the method
	            get.releaseConnection();
	        }
	        
	        
	        
	        int count =0;
	        if (root!=null) {
	        	root = root.getChild("files.index");
	            //for (Iterator it = root.getChildren().iterator();it.hasNext() && ++count<50;) {
	        	for (Iterator it = root.getChildren().iterator();it.hasNext();) {
	            	Element file = (Element)it.next();
//	            	String path = file.getAttributeValue("path").substring(4);
	            	String path = file.getAttributeValue("path");

	            	get = new GetMethod("http://data.icecat.biz/"+path);
	            	try {
	    	            // execute the GET
	    	            int status = client.executeMethod( get );

	    	            // print the status and response
	    	            String resultXML  =get.getResponseBodyAsString();

	    	            SAXBuilder builder = new SAXBuilder();
	    	            Document doc = builder.build(new StringReader(resultXML));
	    	            root = doc.getRootElement();
	    	           
	    	            parseProduct(root);
	    	            

	    	        }catch (Exception e) {
	    	        	logger.error(e);
	    	        }finally {
	    	        
	    	            // release any connection resources used by the method
	    	            get.releaseConnection();
	    	        }
	    	        
	            }
	        }
	        parseAdditionalCategoryData(catRoot);

	}

	/**
	 * @return
	 * @throws IOException
	 * @throws HttpException
	 * @throws JDOMException
	 */
	private Element getCategoryElement() throws IOException, HttpException,
			JDOMException {
		HttpClient categoryclient = new HttpClient();
		GetMethod getCategories = new GetMethod("http://data.icecat.biz/categories.xml");
		getCategories.setDoAuthentication(false);
		int catStatus = categoryclient.executeMethod(getCategories);
		String catXML = getCategories.getResponseBodyAsString();
		SAXBuilder catBuilder = new SAXBuilder();
		Document catDoc = catBuilder.build(new StringReader(catXML));
		Element catRoot = catDoc.getRootElement();
		return catRoot;
	}

	Set<String> features = new HashSet<String>();
	
	private void parseProduct(Element root) {
		
		parseCategories(root);
		
		//parseFeatures(root);
		
//		try {
//			XPath xpath =  XPath.newInstance("/ICECAT-interface/Product");
//			List nodes = xpath.selectNodes(root);
//			for (Iterator it = nodes.iterator();it.hasNext();) {
//				createProduct((Element)it.next());
//			}
//		} catch (JDOMException e) {
//			logger.error(e);
//		}
		
		
		
		
	}

	private void parseAdditionalCategoryData(Element catRoot) {
		try {
			XPath xpath =  XPath.newInstance("//Category");
			List nodes = xpath.selectNodes(catRoot);
			for (Iterator it = nodes.iterator();it.hasNext();) {
				Element cat = (Element)it.next();
				parseAdditionalCategory(cat);
			}
		} catch (Exception e) {
			logger.error(e);
		}
		
	}

	private void parseAdditionalCategory(Element cat) throws Exception {
		ClassMapper cl = new ClassMapper();

		String name = getAttributeValue(cat, "Name[@langid=1]/@Value");

		if (name==null || name.trim().equals(""))
			return;
		URI category = createElement(connection, Vocabulary.CLASS, name);
		addToOntologies(category);

		cl.insertLabel(connection, category, formatLabel(name), Vocabulary.ENGLISH);
		
		ArrayList<TranslationContainer> translations = new ArrayList<TranslationContainer>();
		translations.add(new TranslationContainer(getAttributeValue(cat, "Name[@langid=2]/@Value"),"nl"));
		translations.add(new TranslationContainer(getAttributeValue(cat, "Name[@langid=3]/@Value"),Vocabulary.FRENCH));
		translations.add(new TranslationContainer(getAttributeValue(cat, "Name[@langid=4]/@Value"),Vocabulary.GERMAN));
		translations.add(new TranslationContainer(getAttributeValue(cat, "Name[@langid=5]/@Value"),Vocabulary.ITALIAN));
		translations.add(new TranslationContainer(getAttributeValue(cat, "Name[@langid=6]/@Value"),Vocabulary.SPANISH));
		cl.insertTranslations(connection, category, translations);
		cl.insertComment(connection, category, getAttributeValue(cat, "Description[@langid=1]/@Value"), Vocabulary.ENGLISH);
		ArrayList<String> images = new ArrayList<String>();
		images.add(getAttributeValue(cat, "@LowPic"));
		cl.insertImages(connection, category, images);
		
		XPath parentPath = XPath.newInstance("ParentCategory/Names/Name[@langid=1]");
		List nodes = parentPath.selectNodes(cat);
		ArrayList<URI> parents = new ArrayList<URI>();
		for (Iterator it = nodes.iterator();it.hasNext();) {
//			Text at = (Text)it.next();
//			at.

			Element el = (Element)it.next();

			if (el.getText()==null || el.getText().trim().equals(""))
				continue;
			URI parent = createElement(connection, Vocabulary.CLASS, el.getText());
			cl.insertLabel(connection, parent, formatLabel(el.getText()), Vocabulary.ENGLISH);
			addToOntologies(parent);
			parents.add(parent);
		}
		cl.insertSubclassOf(connection, category, parents);
		
	}

	private String getAttributeValue(Element cat,String path) throws Exception {
		XPath xpath =  XPath.newInstance(path);
		Attribute at = (Attribute)xpath.selectSingleNode(cat);
		if (at==null)
			return null;
		return at.getValue();
	}
	
	private void createProduct(Element root) {

		
		ClassMapper mapper = new ClassMapper();
		try {
			currentSubject = createElement(connection, Vocabulary.CLASS, root.getAttributeValue("Name"));
			mapper.insertLabel(connection, currentSubject, formatLabel(root.getAttributeValue("Name")), Vocabulary.ENGLISH);
			addToOntologies(currentSubject);
			//parseObjects(root, "@Name", RDFS.LABEL);
			
			
			parseComment(root, "ProductDescription[@langid=1]/@LongDesc");
			
			//SEEALSO!

			ArrayList<URI> uris = parseAttributes(root, "ProductFeature/Feature/Name[@langid=1]/@Value", Vocabulary.DATATYPEPROPERTY);
			mapper.insertAllPropertiesHavingClassInDomain(connection, currentSubject, uris);
			//parseObjects(root, "ProductFeature/@Value", Vocabulary.ISINSTANCEOF);
		} catch (Exception e) {
			logger.error(e);
		}
		
		
	}

	
	private void parseComment(Element root, String xp) {
		
		try {
			ClassMapper mapper = new ClassMapper();
			XPath xpath =  XPath.newInstance(xp);
			List nodes = xpath.selectNodes(root);
			for (Iterator it = nodes.iterator();it.hasNext();) {
				Attribute at = (Attribute)it.next();
				String value = at.getValue();

				try {
					mapper.insertComment(connection, currentSubject, value, Vocabulary.ENGLISH);
				} catch (Exception e) {
					logger.error("error creating: comment in "+currentSubject.stringValue(),e);
				}
			}
		} catch (JDOMException e) {
			logger.error(e);
		}
		
	}
	
	/**
	 * @param root
	 * @param xp
	 * @param p
	 */
	private ArrayList<URI> parseAttributes(Element root, String xp, URI p) {
		ArrayList<URI> uris = new ArrayList<URI>();
		try {
			
			XPath xpath =  XPath.newInstance(xp);
			List nodes = xpath.selectNodes(root);
			for (Iterator it = nodes.iterator();it.hasNext();) {
				Attribute at = (Attribute)it.next();
				String value = at.getValue();
				logger.debug(p.stringValue()+": "+at.getValue());
				try {
					uris.add(createElement(connection, Vocabulary.DATATYPEPROPERTY, value));
				} catch (Exception e) {
					logger.error("error creating: "+p.stringValue()+" in "+currentSubject.stringValue(),e);
				}
			}
		} catch (JDOMException e) {
			logger.error(e);
		}
		return uris;
	}

	/**
	 * @param root
	 */
	private void parseFeatures(Element root) {
		try {
			XPath xpath =  XPath.newInstance("//Product/CategoryFeatureGroup/FeatureGroup/Name[@langid=1]/@Value");
			List nodes = xpath.selectNodes(root);
			ArrayList<URI> uris = new ArrayList<URI>();
			ClassMapper mapper = new ClassMapper();
			for (Iterator it = nodes.iterator();it.hasNext();) {
				Attribute at = (Attribute)it.next();
				String feature = at.getValue();

				
				try {
					URI property = createElement(connection, Vocabulary.DATATYPEPROPERTY, feature);
					uris.add(property);
					mapper.insertLabel(connection, property, formatLabel(feature), Vocabulary.ENGLISH);
					//connection.add(currentSubject, Vocabulary.DOMAIN,property);
					
					addToOntologies(property);
				}
				catch (Exception e) {
					logger.error("Error adding feature "+feature+" to "+currentSubject.stringValue(),e);
				}
			}
			mapper.insertAllPropertiesHavingClassInDomain(connection, currentSubject, uris);
		} catch (Exception e) {
			logger.error(e);
		}
	}

	private void parseCategories(Element root) {
		try {
			XPath xpath =  XPath.newInstance("//Product/Category/Name[@langid=1]/@Value");
			List nodes = xpath.selectNodes(root);
			for (Iterator it = nodes.iterator();it.hasNext();) {
				Attribute at = (Attribute)it.next();
				String category = at.getValue();
				if (category==null || category.trim().equals("")) {
					logger.info("we got a problem");
					continue;
				}
				logger.debug("Category: "+category);
				ClassMapper mapper = new ClassMapper();
				try {

//					currentOntology = createElement(connection, Vocabulary.ONTOLOGYMODULE, category);
//					mapper.insertLabel(connection, currentOntology, category, Vocabulary.ENGLISH);
					currentSubject = createElement(connection, Vocabulary.CLASS, category);
					mapper.insertLabel(connection, currentSubject, formatLabel(category), Vocabulary.ENGLISH);
					addToOntologies(currentSubject);
				} catch (Exception e) {
					logger.error("error creating: "+category+" "+e.getMessage(),e);
					currentSubject = new URIImpl(Vocabulary.CLASS_NAMESPACE+category);
				}
				parseFeatures(root);
			}
		} catch (JDOMException e) {
			logger.error(e);
		}
	}

	/**
	 * @throws RepositoryException
	 */
	private void addToOntologies(URI subject) throws RepositoryException {
//		connection.add(subject, Vocabulary.BELONGSTOONTOLOGYMODULE,currentOntology);
		connection.add(subject, Vocabulary.BELONGSTOONTOLOGYMODULE,icecat);
	}

	private HttpClient getLoggedInClient() {
		HttpClient client = new HttpClient();
		UsernamePasswordCredentials buechseCredentials = new UsernamePasswordCredentials("sti2", "rzJT1F");
		UsernamePasswordCredentials ictCredentials = new UsernamePasswordCredentials("sti-innsbruck", "ELTtmU");
		UsernamePasswordCredentials credentials = ictCredentials;
		client.getState().setCredentials(
	            new AuthScope("data.icecat.biz", 80, "Open ICEcat XML repository"),
	            credentials
	        );
		return client;
	}

	public URI createElement( RepositoryConnection con, URI type, String label ) throws Exception {

		URI ret = null;
		
		
		
		
		try {

			label=checkLabelFormat( label );
			if (label==null) return null;
			label = label.trim();

				ret = AbstractDataConnector.createURI( type, label );		
		
				
				//check wether the id exists allready in the sesame data store
				String query = "SELECT x FROM {<" + ret.stringValue() + ">} p {x} LIMIT 1";
				TupleQuery tupleQuery = con.prepareTupleQuery(QueryLanguage.SERQL, query);
				TupleQueryResult result = tupleQuery.evaluate();
				
				try {
					if ( result.hasNext() ){
						return ret;
					}
				} finally {
					result.close();
				}			
	
				con.add( ret, RDF.TYPE, type );
				
				//insert hasVersionDateTime				
				con.add( ret, Vocabulary.HASVERSIONDATETIME, new LiteralImpl( HelperFunctions.convertIntoString( new GregorianCalendar().getTimeInMillis() ) ) );
				
			
		}
		catch( Exception e ) {
			
			logger.error(e);
			
			
			
			ret = null;
		}
		
		return ret;
	}
	
	protected String checkLabelFormat( String newEntry ) {

		if( newEntry==null || newEntry.equals("")) {
			return null;
		}
		else {
			
			/*
			if( newEntry.contains( " " ) ) {
				this.addExceptionLabel( new WrongFormatException( "Spaces are not allowed!" ) );
			}
			*/
			newEntry = newEntry.replace('"', '_');
			newEntry = newEntry.replace('\'', '_');
			newEntry = newEntry.replace('\\', '_');
			newEntry = newEntry.replace('/', '_');
			newEntry = newEntry.replace(' ', '_');
			return newEntry;
		}
	}
	
	protected String formatLabel( String newEntry ) {

		if( newEntry==null || newEntry.equals("")) {
			return null;
		}
		else {
			
			/*
			if( newEntry.contains( " " ) ) {
				this.addExceptionLabel( new WrongFormatException( "Spaces are not allowed!" ) );
			}
			*/
			newEntry = newEntry.replace('"', ' ');
			newEntry = newEntry.replace('\'', ' ');
			newEntry = newEntry.replace('\\', '-');
			newEntry = newEntry.replace('/', '-');
			return newEntry;
		}
	}
	
	
}
